# -*- coding:utf-8 -*-

import urllib
from BeautifulSoup import BeautifulSoup

# store the link Object
arr = []
# store the link String
arrString = []
# init id = 0, the root link is 0
linkID = 0

# Define the LinkItem Class
class LinkItem:
    def __init__(self):
        self.link = ""
        self.id = 0
        self.parentid = 0

# Find all urls in one URL
def lookforurl(linkItem):
    # auto increase ID
    global  linkID
    linkID += 1
    # Get page content
    content = urllib.urlopen(linkItem.link).read()
    # Make it --> BeautifulSoup
    try:
        soup = BeautifulSoup(content)
    except Exception, e:
        return
    # Find A tag items
    for link in soup.findAll("a"):
        # Find href string
        linkString = str(link.get("href"))
        # Not None
        if linkString is not None:
            # Remove space
            linkStringNoSpace = linkString.strip()

            # re-makeup the link
            if linkStringNoSpace.startswith('/'):
                linkStringNoSpace = "http://www.21cake.com" + linkStringNoSpace
            if linkStringNoSpace.startswith('javascript'):
                linkStringNoSpace = ""
            if not linkStringNoSpace.startswith('http'):
                linkStringNoSpace = "http://www.21cake.com/" + linkStringNoSpace
            if linkStringNoSpace == "http://www.21cake.com/":
                linkStringNoSpace = ""
            if linkStringNoSpace == "http://www.21cake.com/index.html":
                linkStringNoSpace = ""
            if linkStringNoSpace == "http://www.21cake.com/#":
                linkStringNoSpace = ""
            if linkStringNoSpace == "http://www.21cake.com":
                linkStringNoSpace = ""
            # Add link to array if new
            if linkStringNoSpace != "":
                if linkStringNoSpace not in arrString:
                    tempLinkItem = LinkItem()
                    tempLinkItem.id = linkID
                    tempLinkItem.link = linkStringNoSpace
                    tempLinkItem.parentid = linkItem.id
                    arr.append(tempLinkItem)
                    arrString.append(linkStringNoSpace)
                    # For Debug
                    print "link: " + tempLinkItem.link + " id: " + str(tempLinkItem.id) + " parentid: " + str(tempLinkItem.parentid)
                    # LOOP
                    lookforurl(tempLinkItem)



# Main Entrance, init the root link item...
rootLink = LinkItem()
rootLink.id = 0
rootLink.link = "http://www.21cake.com"
rootLink.parentid = 0;

lookforurl(rootLink)



print "COUNT: " + str(len(arr))


